# import necessary Libraries
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
import ast
import json
import argparse
import os
# Read Datasets
credit_data=pd.read_csv("data/credits.csv")
# Read Meta dataset
meta=pd.read_csv("data/movies_metadata.csv")
keywords = pd.read_csv('data/keywords.csv')

# Create a function which convert to a list of genres
def make_genresList(x):
    gen = []
    st = " "
    for i in x:
        if i.get('name') == 'Science Fiction':
            scifi = 'Sci-Fi'
            gen.append(scifi)
        else:
            gen.append(i.get('name'))
    if gen == []:
        return np.NaN
    else:
        return (st.join(gen))
    
# Create a function which extract first actor from "cast" column
def get_actor1(x):
    casts=[]
    for i in x:
        casts.append(i.get("name"))
        if casts==[]:
            return np.Nan
        else:
            return (casts[0])
        
# Create a function which extract second actor from "cast" column
def get_actor2(x):
    casts = []
    for i in x:
        casts.append(i.get('name'))
    if casts == [] or len(casts)<=1:
        return np.NaN
    else:
        return (casts[1])

# Create a function which extract third actor from "cast" column
def get_actor3(x):
    casts = []
    for i in x:
        casts.append(i.get('name'))
    if casts == [] or len(casts)<=2:
        return np.NaN
    else:
        return (casts[2])
    
# Create a function which extract Director name from "crew" column
def get_directors(x):
    dt = []
    st = " "
    for i in x:
        if i.get('job') == 'Director':
            dt.append(i.get('name'))
    if dt == []:
        return np.NaN
    else:
        return (st.join(dt))
    
def clean(credit_data,meta,keywords):
    # Extract years from " release_date "
    meta["release_date"]=pd.to_datetime(meta["release_date"],errors="coerce")
    meta['year'] = meta['release_date'].dt.year
    meta["year"].value_counts().sort_index()
    # In meta dataset Extract 'genres','id','title','year' of 2017 movies
    new_meta=meta.loc[meta.year==2017,['genres','id','title','year']]
    # Convert Id column datatype to int
    new_meta["id"]=new_meta["id"].astype(int)
    data = pd.merge(new_meta, credit_data, on='id')
    pd.set_option('display.max_colwidth', 75)
    data["genres"]=data["genres"].map(lambda x : ast.literal_eval(x))
    data["cast"]=data["cast"].map(lambda x : ast.literal_eval(x))
    data["crew"]=data["crew"].map(lambda x : ast.literal_eval(x))
    data['genres_list'] = data['genres'].map(lambda x: make_genresList(x))
    data['actor_1_name'] = data['cast'].map(lambda x: get_actor1(x))
    data['actor_2_name'] = data['cast'].map(lambda x: get_actor2(x))
    data['actor_3_name'] = data['cast'].map(lambda x: get_actor3(x))
    data['director_name'] = data['crew'].map(lambda x: get_directors(x))
    movies_data=data.loc[:,['actor_1_name','actor_2_name', 'actor_3_name', 'director_name',"genres_list",'title']]
    movies_data=movies_data.dropna(how="any")
    # Rename columns
    movies_data=movies_data.rename(columns={'genres_list':'genres'})
    movies_data=movies_data.rename(columns={'title':'movie_title'})
    movies_data["movie_title"]=movies_data["movie_title"].str.lower()
    # 使用rename()方法更改列名
    movies_data = movies_data.rename(columns={'movie_title': 'title'})
    meta["title"]=meta["title"].str.lower()
    movies_data["comb"]=movies_data["actor_1_name"]+' '+movies_data["actor_2_name"]+" "+ movies_data["actor_3_name"]+' ' + movies_data["director_name"]
    meta = meta[['id', 'title', 'overview','popularity','runtime','vote_average','vote_count']]
    movies_data = movies_data[['comb','title']]
    movie = pd.merge(meta,movies_data,left_on='title',right_on='title',how ='left')
    # 替换NaN为''
    keywords['keywords'] = keywords['keywords'].fillna('')

    # 定义一个函数来提取关键词的name
    def extract_keywords(keywords_str):
        try:
            keywords_list = ast.literal_eval(keywords_str)
            keywords_names = ' '.join([keyword['name'] for keyword in keywords_list])
            return keywords_names
        except (ValueError, SyntaxError):
            return ''

    # 应用函数提取关键词的name
    keywords['keywords_extracted'] = keywords['keywords'].apply(extract_keywords)
    # 将标题和概述用空格隔开并存储到一个数组（列表）中
    keywords['keywords'] = keywords['keywords_extracted']
    # 替换NaN为''
    keywords['keywords'] = keywords['keywords'].fillna('')

    # 将id列转换为整数类型，以确保合并时的一致性
    movie['id'] = pd.to_numeric(movie['id'], errors='coerce')
    keywords['id'] = pd.to_numeric(keywords['id'], errors='coerce')
    # 替换NaN为''
    movie['title'] = movie['title'].fillna('')
    movie['overview'] = movie['overview'].fillna('')
    # 删除无效的id行
    # 删除重复的id，保留第一个
    movie = movie.drop_duplicates(subset='id', keep='first')
    # 删除重复的id，保留第一个
    keywords = keywords.drop_duplicates(subset='id', keep='first')

    # 合并两个DataFrame
    merged_data = pd.merge(movie, keywords,left_on='id',right_on='id',how ='left')
    # 清理 NaN 值，将其替换为空字符串
    merged_data['keywords'] = merged_data['keywords'].fillna(' ')
    # 清理 NaN 值，将其替换为空字符串
    merged_data['comb'] = merged_data['comb'].fillna(' ')
    merged_data['title'] = merged_data['title'].fillna(' ')
    merged_data['overview'] = merged_data['overview'].fillna(' ')
    # 添加新列 'ad' 并将其值设置为 0
    merged_data['ad'] = 1.0
    merged_data.to_csv("updata/merge_data.csv")

def clean1(credit_data,meta,keywords):
    # Extract years from " release_date "
    meta["release_date"]=pd.to_datetime(meta["release_date"],errors="coerce")
    meta['year'] = meta['release_date'].dt.year
    meta["year"].value_counts().sort_index()
    # In meta dataset Extract 'genres','id','title','year' of 2017 movies
    new_meta=meta.loc[meta.year==2017,['genres','id','title','year']]
    # Convert Id column datatype to int
    new_meta["id"]=new_meta["id"].astype(int)
    data = pd.merge(new_meta, credit_data, on='id')
    pd.set_option('display.max_colwidth', 75)
    data["genres"]=data["genres"].map(lambda x : ast.literal_eval(x))
    data["cast"]=data["cast"].map(lambda x : ast.literal_eval(x))
    data["crew"]=data["crew"].map(lambda x : ast.literal_eval(x))
    data['genres_list'] = data['genres'].map(lambda x: make_genresList(x))
    data['actor_1_name'] = data['cast'].map(lambda x: get_actor1(x))
    data['actor_2_name'] = data['cast'].map(lambda x: get_actor2(x))
    data['actor_3_name'] = data['cast'].map(lambda x: get_actor3(x))
    data['director_name'] = data['crew'].map(lambda x: get_directors(x))
    movies_data=data.loc[:,['actor_1_name','actor_2_name', 'actor_3_name', 'director_name',"genres_list",'title']]
    movies_data=movies_data.dropna(how="any")
    # Rename columns
    movies_data=movies_data.rename(columns={'genres_list':'genres'})
    movies_data=movies_data.rename(columns={'title':'movie_title'})
    movies_data["movie_title"]=movies_data["movie_title"].str.lower()
    # 使用rename()方法更改列名
    movies_data = movies_data.rename(columns={'movie_title': 'title'})
    meta["title"]=meta["title"].str.lower()
    movies_data["comb"]=movies_data["actor_1_name"]+' '+movies_data["actor_2_name"]+" "+ movies_data["actor_3_name"]+' ' + movies_data["director_name"]
    meta = meta[['id', 'title', 'overview','popularity','runtime','vote_average','vote_count']]
    movies_data = movies_data[['comb','title']]
    movie = pd.merge(meta,movies_data,left_on='title',right_on='title',how ='left')
    # 替换NaN为''
    keywords['keywords'] = keywords['keywords'].fillna('')
    print(keywords['keywords'])
    # 定义一个函数来提取关键词的name
    def extract_keywords(keywords_str):
        try:
            keywords_list = ast.literal_eval(keywords_str)
            keywords_names = ' '.join([keyword['name'] for keyword in keywords_list])
            print(keywords_names)
            return keywords_names
        except (ValueError, SyntaxError):
            return ''
    print(keywords['keywords'])
    # 应用函数提取关键词的name
    keywords['keywords_extracted'] = keywords['keywords'].apply(extract_keywords)
    print(keywords['keywords_extracted'] )
    # 将标题和概述用空格隔开并存储到一个数组（列表）中
    keywords['keywords'] = keywords['keywords_extracted']
    # 替换NaN为''
    keywords['keywords'] = keywords['keywords'].fillna('')

    # 将id列转换为整数类型，以确保合并时的一致性
    movie['id'] = pd.to_numeric(movie['id'], errors='coerce')
    keywords['id'] = pd.to_numeric(keywords['id'], errors='coerce')
    # 替换NaN为''
    movie['title'] = movie['title'].fillna('')
    movie['overview'] = movie['overview'].fillna('')
    # 删除无效的id行
    # 删除重复的id，保留第一个
    movie = movie.drop_duplicates(subset='id', keep='first')
    # 删除重复的id，保留第一个
    keywords = keywords.drop_duplicates(subset='id', keep='first')

    # 合并两个DataFrame
    merged_data = pd.merge(movie, keywords,left_on='id',right_on='id',how ='left')
    # 清理 NaN 值，将其替换为空字符串
    merged_data['keywords'] = merged_data['keywords'].fillna(' ')
    # 清理 NaN 值，将其替换为空字符串
    merged_data['comb'] = merged_data['comb'].fillna(' ')
    merged_data['title'] = merged_data['title'].fillna(' ')
    merged_data['overview'] = merged_data['overview'].fillna(' ')
    # 添加新列 'ad' 并将其值设置为 0
    merged_data['ad'] = 1.0
    # 目标 CSV 文件路径
    csv_file_path = "updata/merge_data2.csv"

    # 检查文件是否存在
    file_exists = os.path.isfile(csv_file_path)

    # 追加数据到 CSV 文件
    merged_data.to_csv(csv_file_path, mode='a', index=False, header=not file_exists)


    

def json_to_dataframes(json_data):
    """
    将 JSON 数据转换为多个 Pandas DataFrame，并创建一个 credits DataFrame。
    """
    # 解析 JSON 数据
    data = json.loads(json_data)

    # 提取演员信息并转换为 DataFrame
    actors_df = pd.DataFrame(data.get('actors', []))

    # 提取电影的基本信息并转换为 DataFrame
    movie_info = data.get('movie', {})
    
    # 将 genreHubs 移动到 movie 中并重命名为 genres
    genre_hubs = data.pop("genreHubs", [])
    movie_info['genres'] = genre_hubs
    
  # 定义 movies_df 需要的所有列
    movie_columns = [
        'id', 'adult', 'belongs_to_collection', 'budget', 'genres', 'homepage', 'originalLanguage',
        'originalTitle', 'overview', 'popularity', 'posterPath', 'production_companies',
        'production_countries', 'release_date', 'revenue', 'runtime', 'spoken_languages', 'status',
        'tagline', 'title', 'video', 'vote_average', 'vote_count'
    ]
    
    # 默认值设置
    default_values = {
        'popularity': 0,
        'runtime': 0,
        'vote_average': 0,
        'vote_count': 0
    }
    
    # 提取现有 movie 信息，并初始化未提供的列为空值或默认值
    movie_data = {col: movie_info.get(col, default_values.get(col, None)) for col in movie_columns}
    
    # 转换为 DataFrame
    movie_df = pd.DataFrame([movie_data])

    # 获取电影的 id
    movie_id = movie_info.get('id', None)
    
    # 提取关键词信息，添加电影 id 并转换为 DataFrame
    keywords = data.get('keywords', [])
    # for keyword in keywords:
    #     keyword['movie_id'] = movie_id
    # keywords_df = pd.DataFrame(keywords)
    keywords_str = json.dumps(keywords)  # 将关键词列表转换为字符串格式
    keywords_df = pd.DataFrame([{
            'id': movie_id,
            'keywords':keywords_str,
        }])

    # 创建一个 credits DataFrame
    credits_df = pd.DataFrame([{
        'id': movie_id,
        'cast': json.dumps(data.get('actors', [])),
        'crew': []
    }])

    return actors_df, movie_df, keywords_df, credits_df

def main():
    # 设置参数解析器
    parser = argparse.ArgumentParser(description="处理 JSON 数据并将其转换为多个 Pandas DataFrame。")
    
    # 添加参数，用于接收 JSON 数据
    parser.add_argument('json_data', type=str, help='输入的 JSON 数据字符串')
    
    # 解析命令行参数
    args = parser.parse_args()
    
    # 将 JSON 数据转换为 DataFrame
    actors_df, movie_df, keywords_df, credits_df = args.json_data
    
    # 调用 clean1 函数
    clean1(credits_df, movie_df, keywords_df)

if __name__ == "__main__":
    main()
    